{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 3. Statistical Information\n",
    "\n",
    "## Contents\n",
    "- [Configuration](#Configuration)\n",
    "  - [Imports](#Imports)\n",
    "  - [Variables](#Variables)\n",
    "  - [Support functions](#Support-functions)\n",
    "- [Botscore distribution](#Botscore-distribution)\n",
    "- [Daily total traffic](#Daily-total-traffic)\n",
    "- [Tweet type distributions per botscore authorship](#Tweet-type-distributions-per-botscore-authorship)\n",
    "- [Number of users per number of interactions](#Number-of-users-per-number-of-interactions)\n",
    "- [Interactions between humans and bots](#Interactions-between-humans-and-bots)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "from fastprogress import master_bar, progress_bar\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "# Graphic utilities\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(font_scale=0.9)\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_style({'font.family':'monospace'})\n",
    "from pylab import *\n",
    "\n",
    "# Dataframes\n",
    "import pandas as pd\n",
    "from collections.abc import MutableMapping\n",
    "import numpy as np\n",
    "\n",
    "# MongoDB functionality\n",
    "from pymongo.errors import BulkWriteError\n",
    "from pymongo import MongoClient\n",
    "from bson import ObjectId\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If true exports vectorial PDFs instead of JPG.\n",
    "VECTORIAL_FIGURES = True\n",
    "FIG_EXTENSION = \".pdf\" if VECTORIAL_FIGURES else \".jpg\"\n",
    "\n",
    "# Directories where CSV data is stored\n",
    "ROOT_DIR = \"ABOSLUTE_PATH_TO_ROOT_FOLDER\"\n",
    "DATA_DIR = ROOT_DIR + \"data/\"\n",
    "GRAPHICS_DIR = ROOT_DIR + \"graphics/\"\n",
    "\n",
    "# Change path to root\n",
    "os.chdir(ROOT_DIR)\n",
    "\n",
    "try:\n",
    "    os.makedirs(GRAPHICS_DIR)\n",
    "except FileExistsError:\n",
    "    # directory already exists\n",
    "    pass\n",
    "\n",
    "# MongoDB parameters\n",
    "mongoclient = MongoClient('IP_ADDRESS', PORT)\n",
    "db = mongoclient.DATABASE_NAME\n",
    "# It will automatically create the tweets' and users' collections."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Support Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_objid(text):\n",
    "    \"\"\"Makes an ObjectId of 4 bytes\n",
    "    \n",
    "    Keyword arguments:\n",
    "    text -- string to be converted into Object ID\n",
    "    \"\"\"\n",
    "    text = str(text)\n",
    "    if not text.strip():\n",
    "        return None\n",
    "    try:\n",
    "        return ObjectId(text.rjust(24,\"0\"))\n",
    "    except Exception as ex:\n",
    "        print(text, ex)\n",
    "        return None\n",
    "\n",
    "def flatten(d, parent_key='', sep='_'):\n",
    "    \"\"\"Formats MongoDB results\n",
    "    \n",
    "    Keyword arguments:\n",
    "    d -- dictionary with key and uncleaned values\n",
    "    parent_key --\n",
    "    sep --\n",
    "    \"\"\"\n",
    "    items = []\n",
    "    for k, v in d.items():\n",
    "        new_key = parent_key + sep + k if parent_key else k\n",
    "        if isinstance(v, MutableMapping):\n",
    "            items.extend(flatten(v, new_key, sep=sep).items())\n",
    "        else:\n",
    "            items.append((new_key, v))\n",
    "    return dict(items)\n",
    "\n",
    "# color palette\n",
    "colors = [\"#C44E52\", #red\n",
    "          \"#55A868\", #green\n",
    "          \"#FFC400\", #yellow\n",
    "          \"#4C72B0\", #blue\n",
    "          \"#DD8452\", #orange\n",
    "          \"#8172B3\", #purple\n",
    "          \"#64B5CD\", #cyan\n",
    "          \"#937860\", #brown\n",
    "          \"#8C8C8C\", #gray\n",
    "         ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Botscore distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "def load_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and botscore of users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    pipeline = [\n",
    "                {\n",
    "                    '$match': {\n",
    "                        'scores': {\n",
    "                            '$exists': True\n",
    "                        },\n",
    "                    }\n",
    "                }, {\n",
    "                    '$project': {\n",
    "                        '_id': True, \n",
    "                        'scores.scores.universal': True\n",
    "                    }\n",
    "                }\n",
    "            ]\n",
    "    print(\"Query\", end=\" \")\n",
    "    users = user_collection.aggregate(pipeline, allowDiskUse=True)\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    users = list(users)\n",
    "    print(\"OK; Total users:\", len(users))\n",
    "    return users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "users = load_users(db.users)\n",
    "users = [flatten(u) for u in users]\n",
    "df_users = pd.DataFrame(users)\n",
    "df_users.columns = ['uid', 'universal']\n",
    "p75 = np.percentile(df_users.universal,75)\n",
    "p95 = np.percentile(df_users.universal,95)\n",
    "print(\"Limits:\", p75, p95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Humans:',len(df_users[df_users.universal<=p75]))\n",
    "print('Bots:',len(df_users[df_users.universal>=p95]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we round the botscore to discretize\n",
    "df_users.universal = np.round(df_users.universal,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot botscore distribution\n",
    "\n",
    "fig, axs = plt.subplots(3, 1, figsize=(5,10))\n",
    "\n",
    "# subfigure 1\n",
    "g = axs[0]\n",
    "g = sns.distplot(df_users.universal, \n",
    "                 bins=50,\n",
    "                 hist=True, \n",
    "                 kde=False, \n",
    "                 ax=g, \n",
    "                 #label=tweet_type.capitalize(), \n",
    "                 color=colors[3])\n",
    "g.set_ylabel(fontsize=\"small\", ylabel='Number of accounts')\n",
    "g.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "g.title.set_text(\"All users\")\n",
    "\n",
    "# subfigure 2\n",
    "h = axs[1]\n",
    "h = sns.distplot(df_users.universal, \n",
    "                 bins=50,\n",
    "                 hist=True, \n",
    "                 kde=False, \n",
    "                 ax=h, \n",
    "                 #label=tweet_type.capitalize(), \n",
    "                 hist_kws={'cumulative': True},\n",
    "                 color=colors[3])\n",
    "h.set_ylabel(fontsize=\"small\", ylabel='Cumulative number of accounts')\n",
    "h.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "\n",
    "# subfigure 3\n",
    "i = axs[2]\n",
    "i = sns.boxenplot(df_users.universal, \n",
    "                ax=i, \n",
    "                color=colors[3])\n",
    "i.set_ylabel(fontsize=\"small\", ylabel='Score distribution')\n",
    "i.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "\n",
    "\n",
    "# vertical lines for percentiles\n",
    "g_ylims = g.get_ylim()\n",
    "h_ylims = h.get_ylim()\n",
    "i_ylims = i.get_ylim()\n",
    "prs = [50,75,90,95,99]\n",
    "ps = np.percentile(a=df_users.universal, q=prs)\n",
    "for pr,p in zip(prs,ps):\n",
    "    label = str(pr) + \"th Perc.\"\n",
    "    g.vlines(x=p, ymin=g_ylims[0], ymax=g_ylims[1], linewidth = 2, label=label, linestyles='dotted')\n",
    "    h.vlines(x=p, ymin=h_ylims[0], ymax=h_ylims[1], linewidth = 2, linestyles='dotted')\n",
    "    i.vlines(x=p, ymin=i_ylims[0], ymax=i_ylims[1], linewidth = 2, linestyles='dotted')\n",
    "\n",
    "# common legend\n",
    "g.legend(loc=\"upper right\")\n",
    "h.legend(loc=\"lower right\")\n",
    "\n",
    "# figure \n",
    "fig.suptitle(\"Botscore distribution\", y=1.01)\n",
    "plt.tight_layout()\n",
    "plt.savefig(GRAPHICS_DIR + \"Botscores-distribution\" + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daily total traffic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID and date of creation of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':1,'created_at':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = load_tweets(db.tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tweets = pd.DataFrame(tweets)\n",
    "df_tweets['date'] = df_tweets.created_at.dt.date   # we extract the date (year-day-month) from timestamp (created_at)\n",
    "df_tweets.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = min(df_tweets['date'])     # first day\n",
    "end = max(df_tweets['date'])       # last day\n",
    "\n",
    "# count of interactions per day\n",
    "df_dates = pd.DataFrame(df_tweets.groupby('date').agg({'_id':'count'}))\n",
    "df_dates.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot traffic per day, blue color according to traffic volume\n",
    "pal = sns.color_palette(\"Blues_d\", len(df_dates))\n",
    "pal2 = []\n",
    "df_dates2 = df_dates.sort_values(by='_id',axis='index',ascending=False).reset_index().sort_values(by='date',axis='index')\n",
    "for _id in df_dates2.index:\n",
    "    pal2.append(pal[_id])\n",
    "\n",
    "plt.figure(figsize=(15,5))\n",
    "ax=sns.barplot(x=df_dates.date, y=df_dates._id, palette=pal2)\n",
    "ax.xaxis.set_tick_params(rotation=75)\n",
    "plt.tight_layout()\n",
    "plt.xlabel('Date')\n",
    "plt.ylabel('Number of interactions')\n",
    "plt.title(\"Total traffic during collecting period\")\n",
    "plt.savefig(GRAPHICS_DIR + \"total-traffic-timeline\" + FIG_EXTENSION,bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tweet type distributions per botscore authorship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"\n",
    "    Extracts all tweets with their tweet type and author botscore\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    pipeline = [\n",
    "                {\n",
    "                    '$lookup': {\n",
    "                        'from': 'users', \n",
    "                        'localField': 'user_id', \n",
    "                        'foreignField': '_id', \n",
    "                        'as': 'usr'\n",
    "                    }\n",
    "                }, {\n",
    "                    '$unwind': {\n",
    "                        'path': '$usr', \n",
    "                        'preserveNullAndEmptyArrays': False\n",
    "                    }\n",
    "                }, {\n",
    "                    '$match': {\n",
    "                        'usr.scores': {\n",
    "                            '$exists': True\n",
    "                        }\n",
    "                    }\n",
    "                }, {\n",
    "                    '$project': {\n",
    "                        '_id': False, \n",
    "                        'tweet_type': True, \n",
    "                        'user_bot_score': '$usr.scores.scores.universal',#.cap.universal', \n",
    "                    }\n",
    "                }\n",
    "            ]\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.aggregate(pipeline, allowDiskUse=True)\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)  \n",
    "    print(\"OK; Total records:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"DF:\",len(tweets))\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)\n",
    "display(df_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we round the botscore to discretize (for the graphic)\n",
    "df_tweets['user_score_bin'] = np.around(df_tweets.user_bot_score,2)\n",
    "\n",
    "# plot tweet type distributions\n",
    "# each column is a tweet type, each row a type of graphic\n",
    "fig, axs = plt.subplots(3, len(df_tweets.tweet_type.unique()), figsize=(15,10))\n",
    "ax_idx = 0\n",
    "\n",
    "# each iteration creates a column regarding a tweet type\n",
    "grps = df_tweets.groupby(by=['tweet_type'])\n",
    "for tweet_type,grp in grps:\n",
    "    \n",
    "    # first plot of the tweet type column\n",
    "    g = axs[0][ax_idx]  \n",
    "    g = sns.distplot(grp.user_score_bin, \n",
    "                     bins=50,\n",
    "                     hist=True, \n",
    "                     kde=False, \n",
    "                     ax=g, \n",
    "                     label=tweet_type.capitalize(), \n",
    "                     color=colors[ax_idx])\n",
    "    g.set_ylabel(fontsize=\"small\", ylabel='Number of interactions')\n",
    "    g.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "    g.title.set_text(\"Type:\" + tweet_type)\n",
    "    \n",
    "    # second plot of the tweet type column\n",
    "    h = axs[1][ax_idx] \n",
    "    h = sns.distplot(grp.user_score_bin, \n",
    "                     bins=50,\n",
    "                     hist=True, \n",
    "                     kde=False, \n",
    "                     ax=h, \n",
    "                     label=tweet_type.capitalize(), \n",
    "                     hist_kws={'cumulative': True},\n",
    "                     color=colors[ax_idx])\n",
    "    h.set_ylabel(fontsize=\"small\", ylabel='Cumulative number of interactions')\n",
    "    h.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "    \n",
    "    # third plot of the tweet type column\n",
    "    i = axs[2][ax_idx]\n",
    "    i = sns.boxenplot(grp.user_score_bin, \n",
    "                    ax=i, \n",
    "                    color=colors[ax_idx])\n",
    "    i.set_ylabel(fontsize=\"small\", ylabel='Interactions distribution')\n",
    "    i.set_xlabel(fontsize=\"small\", xlabel='User\\'s Universal Score')\n",
    "    \n",
    "    \n",
    "    # vertical lines for percentiles\n",
    "    g_ylims = g.get_ylim()\n",
    "    h_ylims = h.get_ylim()\n",
    "    i_ylims = i.get_ylim()\n",
    "    \n",
    "    prs = [50,75,90,95,99]\n",
    "    ps = np.percentile(df_users.universal, q=prs)\n",
    "    for pr,p in zip(prs,ps):\n",
    "        label = str(pr) + \"th Perc.\"\n",
    "        g.vlines(x=p, ymin=g_ylims[0], ymax=g_ylims[1], linewidth = 2, label=label, linestyles='dotted')\n",
    "        h.vlines(x=p, ymin=h_ylims[0], ymax=h_ylims[1], linewidth = 2, linestyles='dotted')\n",
    "        i.vlines(x=p, ymin=i_ylims[0], ymax=i_ylims[1], linewidth = 2, linestyles='dotted')\n",
    "        \n",
    "    # legends\n",
    "    g.legend(loc=\"upper right\")\n",
    "\n",
    "    ax_idx+=1 \n",
    "\n",
    "# figure parameters\n",
    "fig.suptitle(\"Interactions distributions per tweet type and botscores\", y=1.01)\n",
    "plt.tight_layout()\n",
    "plt.savefig(GRAPHICS_DIR + \"Interactions_distribution_per_scores\" + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of users per number of interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_human_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID of human users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    p75 = 0.23633691139538376   # we consider humans to those users with a botscore equal or smaller to 75th percentile\n",
    "    print(\"Query\", end=\" \")\n",
    "    human_users = user_collection.find({'scores.scores.universal': {'$lte': p75}},\n",
    "                                  {'_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    human_users = list(human_users)\n",
    "    print(\"OK; Total human users:\", len(human_users))\n",
    "    return human_users\n",
    "\n",
    "def load_bot_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID of bot users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    p95 = 0.6908019160064479   # we consider bots to those users with a botscore equal or greater than 95th percentile\n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_users = user_collection.find({'scores.scores.universal': {'$gte': p95}},\n",
    "                                  {'_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_users = list(bot_users)\n",
    "    print(\"OK; Total bot users:\", len(bot_users))\n",
    "    return bot_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID, tweet type and user id of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':1,'tweet_type':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "human_users = load_human_users(db.users)\n",
    "df_human_users = pd.DataFrame(human_users)\n",
    "display(df_human_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_users = load_bot_users(db.users)\n",
    "df_bot_users = pd.DataFrame(bot_users)\n",
    "display(df_bot_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time   # getting human tweets\n",
    "df_human_tweets = df_tweets[df_tweets['user_id'].isin(df_human_users['_id'])]\n",
    "df_human_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_human_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time    # getting bot tweets\n",
    "df_bot_tweets = df_tweets[df_tweets['user_id'].isin(df_bot_users['_id'])]\n",
    "df_bot_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_bot_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\"\"\"\n",
    "two subplots sharing Y-axis will show how many users there are per number of created interactions\n",
    "we consider humans and bots users\n",
    "\"\"\"\n",
    "fig, axs = plt.subplots(1, 2, figsize=(16,3), sharey=True)\n",
    "\n",
    "# first iteration for human tweets, second for bots\n",
    "case='Humans'\n",
    "j=0\n",
    "for data in [df_human_tweets,df_bot_tweets]:\n",
    "    # count the number of interactions generated by each user (per tweet type)\n",
    "    counts = pd.DataFrame(data.groupby(['tweet_type','user_id']).agg({'user_id':'count'}))\n",
    "    counts.columns = ['interactions']\n",
    "    counts = pd.DataFrame(counts.to_records())\n",
    "    \n",
    "    # count the number of users by number of interactions (per tweet type)\n",
    "    d = pd.DataFrame(counts.groupby(['tweet_type','interactions']).agg({'user_id':'count'}))\n",
    "    d.columns = ['users_count']\n",
    "    d = pd.DataFrame(d.to_records())\n",
    "\n",
    "    # plot\n",
    "    g = axs[j]\n",
    "    g = sns.scatterplot(data=d, x='interactions', y='users_count', hue='tweet_type', ax=g)\n",
    "    if case=='Humans':\n",
    "        g.set_xlim(left=0, right=1000)\n",
    "    else:\n",
    "        g.set_xlim(left=0, right=600)\n",
    "    g.set_title(case)\n",
    "    g.set_yscale('log')\n",
    "    #g.legend(loc='upper right', title=\"Tweet type\")\n",
    "    g.set_xlabel(\"Number of interactions\")\n",
    "    g.set_ylabel(\"Number of users\")\n",
    "    g.tick_params(axis='y',labelleft=True)\n",
    "    case='Bots'\n",
    "    j=j+1\n",
    "\n",
    "# figure parameters\n",
    "plt.tight_layout()\n",
    "plt.savefig(GRAPHICS_DIR+\"users-per-number-of-interactions\" + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interactions between humans and bots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "def load_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and botscore of users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    pipeline = [\n",
    "                {\n",
    "                    '$match': {\n",
    "                        'scores': {\n",
    "                            '$exists': True\n",
    "                        }\n",
    "                    }\n",
    "                }, {\n",
    "                    '$project': {\n",
    "                        '_id': True, \n",
    "                        'botscore':'$scores.scores.universal'\n",
    "                    }\n",
    "                }\n",
    "            ]\n",
    "    print(\"Query\", end=\" \")\n",
    "    users = user_collection.aggregate(pipeline, allowDiskUse=True)\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    users = list(users)\n",
    "    print(\"OK; Total users:\", len(users))\n",
    "    return users\n",
    "\n",
    "def load_referencing_tweets(tweet_collection):\n",
    "     \"\"\"Extracts the ObjectID, tweet type, user id, retweeted/quoted user id and replied user id of retweets, replies and quotes\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    referencing_tweets = list(tweet_collection.find(\n",
    "    filter={\n",
    "        'tweet_type': \n",
    "                {'$in' : ['retweet',\n",
    "                          'reply',\n",
    "                          'quote']}},\n",
    "    projection={'_id':1, \n",
    "                'tweet_type':1,\n",
    "                'user_id':1, \n",
    "                'retweet_or_quote_user_id':1, \n",
    "                'in_reply_to_user_id':1},\n",
    "    ))\n",
    "\n",
    "    print(\"Referencing tweets extracted:\",len(referencing_tweets))\n",
    "    return referencing_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "users = load_users(db.users)\n",
    "df_users = pd.DataFrame(users)\n",
    "p75 = np.percentile(df_users.botscore,75)\n",
    "p95 = np.percentile(df_users.botscore,95)\n",
    "display(df_users.sample(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_referencing_tweets(db.tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\"\"\"\n",
    "Each tweet is complemented with the botscore of the active user (user who creates the interaction) and the botscore of the passive user (user who creates the referenced interaction)\n",
    "Note that, in this analysis, we should only consider those retweets, replies and quotes with known active and passive user botscore.\n",
    "\"\"\"\n",
    "df_tweets = pd.DataFrame(tweets)\n",
    "df_tweets.columns = ['tweet_id','passive_id_reply','passive_id','tweet_type','active_id']\n",
    "df_tweets.passive_id.fillna(df_tweets.passive_id_reply,inplace=True)\n",
    "df_tweets.drop(columns=['passive_id_reply'],inplace=True)\n",
    "df_tweets = df_tweets.join(df_users.set_index('_id'), on='passive_id')\n",
    "df_tweets.columns = ['tweet_id','passive_id','tweet_type','active_id','passive_id_botscore']\n",
    "df_tweets = df_tweets.join(df_users.set_index('_id'), on='active_id')\n",
    "df_tweets.columns = ['tweet_id','passive_id','tweet_type','active_id','passive_id_botscore','active_id_botscore']\n",
    "print('Refering interactions (replies,quotes,retweets):',len(df_tweets))\n",
    "df_tweets = df_tweets.dropna(axis='index').reset_index(drop=True)\n",
    "print('Refering interactions (replies,quotes,retweets) with botscore in source and destination:',len(df_tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time  \n",
    "\"\"\"\n",
    "Given the active and passive botscores of each tweet, we derive the active and passive categories:\n",
    "- A botscore from 0 to 75th percentile is mapped to human category\n",
    "- A botscore from 75th percentile to 95th percentile is mapped to unclear category\n",
    "- A botscore from 95th percentile to 1 is mapped to bot category\n",
    "\"\"\"\n",
    "df_tweets['active_user_category'] = pd.cut(df_tweets.active_id_botscore, \n",
    "                            bins=[0, p75, p95, 1], \n",
    "                            right=True, \n",
    "                            include_lowest=True, \n",
    "                            labels=['Human', 'Unclear', 'Bot'])\n",
    "\n",
    "\n",
    "df_tweets['passive_user_category'] = pd.cut(df_tweets.passive_id_botscore, \n",
    "                            bins=[0, p75, p95, 1], \n",
    "                            right=True, \n",
    "                            include_lowest=True, \n",
    "                            labels=['Human', 'Unclear', 'Bot'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interactions with uncertain users\n",
    "print(\"Replies/Retweets/Quotes involving unclear users:\",len(df_tweets[(df_tweets.active_user_category=='Unclear') | (df_tweets.passive_user_category=='Unclear')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We remove those tweets involving uncertain users: we consider only human and bot interactions\n",
    "df_tweets = df_tweets[df_tweets.active_user_category!=\"Unclear\"]\n",
    "df_tweets.active_user_category = df_tweets.active_user_category.astype(str).astype('category')\n",
    "df_tweets = df_tweets[df_tweets.passive_user_category!=\"Unclear\"]\n",
    "df_tweets.passive_user_category = df_tweets.passive_user_category.astype(str).astype('category')\n",
    "df_tweets.reset_index(drop=True, inplace=True)\n",
    "print(\"Replies/Retweets/Quotes involving humans and bots users:\",len(df_tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We calculate the total count and log count of the combinations: passive category, active category and tweet type\n",
    "df_heatplot = df_tweets[['active_user_category','passive_user_category','tweet_type']]\n",
    "df_heatplot['count'] = 1\n",
    "df_heatplot = df_heatplot.groupby(['active_user_category','passive_user_category','tweet_type']).count()\n",
    "df_heatplot['log_count'] = np.log10(df_heatplot['count'])\n",
    "#df_heatplot.drop(columns=\"count\",inplace=True)\n",
    "df_heatplot.reset_index(inplace=True)\n",
    "df_heatplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_heatmap(data,fmt,title):\n",
    "    \"\"\"\n",
    "    Creates a heatmap \n",
    "    \n",
    "    Keywords argument:\n",
    "    data -- DataFrame with records\n",
    "    fmt -- format of cell's annotations\n",
    "    title -- title of the heatmap\n",
    "    \"\"\"\n",
    "    ax = sns.heatmap(data=data,cmap=sns.light_palette(\"purple\"),annot=True,linecolor='black',fmt=fmt)\n",
    "    bottom, top = ax.get_ylim()\n",
    "    ax.set_ylim(bottom + 0.5, top - 0.5)\n",
    "    ax.set_xlabel(\"Passive user category\")\n",
    "    ax.set_ylabel(\"Active user category & tweet type\")\n",
    "    ax.set_title(\"Interaction volumes (\"+title+\")\")\n",
    "\n",
    "    boxes = [\n",
    "        #left\n",
    "        {'x': 0, 'y': 0, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "        {'x': 0, 'y': 2, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "        {'x': 0, 'y': 4, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "        #right\n",
    "        {'x': 1, 'y': 0, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "        {'x': 1, 'y': 2, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "        {'x': 1, 'y': 4, 'w': 1, 'h': 2, 'color': \"#1b5e20\"},\n",
    "    ]\n",
    "\n",
    "    for box in boxes:\n",
    "        ax.hlines(y=box['y'], xmin=box['x'], xmax=box['x']+box['w'], colors=box['color'], linewidth = 3)\n",
    "        ax.hlines(y=box['y']+box['h'], xmin=box['x'], xmax=box['x']+box['w'], colors=box['color'], linewidth = 2)\n",
    "\n",
    "        ax.vlines(x=box['x'], ymin=box['y'], ymax=box['y']+box['h'], colors=box['color'], linewidth = 3)\n",
    "        ax.vlines(x=box['x']+box['w'], ymin=box['y'], ymax=box['y']+box['h'], colors=box['color'], linewidth = 3)\n",
    "\n",
    "    #plt.tight_layout()\n",
    "    plt.savefig(GRAPHICS_DIR + \"interaction-volumes-\" + title + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataframe management for plotting heatmaps\n",
    "index=pd.MultiIndex.from_product([['Bot','Human'],['Bot','Human'],['quote','reply','retweet']],names=['Active','Passive','TweetType'])\n",
    "df_heatplot.set_index(index,inplace=True)\n",
    "df_heatplot.drop(columns=['active_user_category','passive_user_category','tweet_type'],inplace=True)\n",
    "df_heatplot_count = pd.pivot_table(df_heatplot,values='count',index=['TweetType','Active'],columns=['Passive'])\n",
    "df_heatplot_log = pd.pivot_table(df_heatplot,values='log_count',index=['TweetType','Active'],columns=['Passive'])\n",
    "\n",
    "# plot heatmaps\n",
    "generate_heatmap(df_heatplot_count,'d','count')\n",
    "generate_heatmap(df_heatplot_log,'.2g','log')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
